/*
 *
 *
Copyright (c) 1998-2001 Aristar, Inc. All rights reserved.

Redistribution and use in source and binary forms, with or without modification,
are permitted provided that the following conditions are met:

1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.

2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.

3. The end-user documentation included with the redistribution, if any,
must include the following acknowledgment:

"This product includes software developed by Aristar, Inc. (http://www.aristar.com/)."

Alternately, this acknowledgment may appear in the software itself, if and
wherever such third-party acknowledgments normally appear.

4. The names "Dialect" and "Aristar, Inc." must not be used to endorse or
promote products derived from this software without prior written permission.
For written permission, please contact info@aristar.com.

5. Products derived from this software may not be called "Dialect", nor may
"Dialect" appear in their name, without prior written permission of Aristar, Inc.

THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESSED OR IMPLIED WARRANTIES,
INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY
AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL
ARISTAR, INC. OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

This software consists of voluntary contributions made by many individuals
on behalf of Aristar, Inc.  For more information on Aristar, Inc.,
please see <http://www.aristar.com/>.

Portions of this software are based upon software freely available and
written by individuals not associated with Aristar, Inc.  Copyrights are
maintained in the relevant source code files.
 *
 *
 */

#include "stdafx.h"

#include "DConfigGlobals.h"
#include <stdlib.h>
#include <string.h>
#include <ctype.h>
#include "CByteCode.h"
#include "CDebug.h"
#include "CValue.h"	// for Serialize help

///////////////////////////////////
#if defined(USING_MFC) && defined(_DEBUG)
#define new DEBUG_NEW
#undef THIS_FILE
static char THIS_FILE[] = __FILE__;
#endif
///////////////////////////////////

#if defined(__WINCE__) //&& (UNDER_CE < 300)  something strange happens if we don't redefine these under 3.0
#	ifndef EOF
#		define EOF			(char)0xFF
#	endif
#	define isdigit(x)	(((x) >= '0') && ((x) <= '9'))
#	define isalpha(x)	((((x) >= 'a') && ((x) <= 'z')) || (((x) >= 'A') && ((x) <= 'Z')))
#	define isalnum(x)	(isalpha(x) || isdigit(x))
#else
#	include <stdio.h>
#endif

//const char cEOF = (char)EOF; // Just testing

///////////////////////////////////////////////////////////
//	Helper function...
///////////////////////////////////////////////////////////

TString TokToString(const CByteCode& token)
{
	// Useful in error message creation
	TString str;
	switch ((TokenTypes)token) {
		case tokIdent:
			str = (LPCTSTR)token;
			break;
		case tokFloat:
			str.Format(_T("%f"), (float)token);
			break;
		case tokInteger:
			str.Format(_T("%d"), (int)token);
			break;
		default:
			str = TokenTypeToString((TokenTypes)token);
	}
	return str;
}

static TString GetFullPath(TString filename)
{
	// Extract the full path from the filename, the return value
	// includes the final directory separator character

#if defined(__WINCE__)
	int pos = filename.ReverseFind(DIR_SEPARATOR);
	if (pos >= 0) {
		return filename.Left(pos + 1);
	}
	else {
		return DIR_SEPARATOR;	// Can this happen under CE?  Shouldn't...
	}
#elif defined(__WINNT__)
	LPTSTR lpFilePart;
	TCHAR buf[MAX_PATH];
	DWORD count = GetFullPathName(filename, MAX_PATH, buf, &lpFilePart);
	*lpFilePart = 0;	// NULL-off the filename
	return buf;
#else
	ASSERT(false);
#endif
}

///////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////
//	Compiling text into the m_CodeBuffer
///////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////

// Compile ASCII text buffer
bool CByteCode::_Compile(LPCSTR text, LPCTSTR moduleName, TString path)
{
	m_ParentModule = moduleName;
	m_Path = path;

	// Do compile here... Rewind() was called in _StartNow()
	return _StartNow(text, strlen(text));
}

// Compile Unicode text buffer
bool CByteCode::_Compile(LPCWSTR text, LPCTSTR moduleName, TString path)
{
	m_ParentModule = moduleName;
	m_Path = path;

	// Thunk this wide char text to 8-bit ASCII
	// Maybe someday we will do this on the fly during compilation...
	int textLength = wcslen(text);
	char *buf = new char[textLength + 1];
	for (int idx = 0; idx < textLength + 1; ++idx) {
		// This loop also copies the NULL terminator (as it should)
		buf[idx] = (char)(text[idx]);
	}

	// Rewind() was called in _StartNow()
	bool bResult = _StartNow(buf, textLength);
	delete[] buf;
	return bResult;
}

bool CByteCode::CheckSpace(int bytes)
{
	if (m_Index + bytes >= m_CodeBufferLength - 1) {
		m_CodeBufferLength += bytes + 2048;
		m_CodeBuffer = (U8*)realloc(m_CodeBuffer, m_CodeBufferLength);
		if (m_CodeBuffer == NULL) {
			m_LastError = _T("Failed to allocate more memory for code buffer");
			return false;
		}
	}
	return true;
}

bool CByteCode::Store(TokenTypes x)
{
	if (!CheckSpace(1)) return false;
	m_CodeBuffer[m_Index++] = x;
	return true;
}

bool CByteCode::Store(int x)
{
	if (!CheckSpace(4)) return false;
	// Need to ensure byte-endian independence
	m_CodeBuffer[m_Index++] = (U8)((x & 0xFF000000) >> 24);
	m_CodeBuffer[m_Index++] = (U8)((x & 0x00FF0000) >> 16);
	m_CodeBuffer[m_Index++] = (U8)((x & 0x0000FF00) >>  8);
	m_CodeBuffer[m_Index++] = (U8) (x & 0x000000FF);
	return true;
}

void CByteCode::Read(int& x)
{
	x = (((U8)(m_CodeBuffer[m_Index])     << 24) |
		 ((U8)(m_CodeBuffer[m_Index + 1]) << 16) |
		 ((U8)(m_CodeBuffer[m_Index + 2]) <<  8) |
		  (U8) m_CodeBuffer[m_Index + 3]);

	m_Index += 4;
}

bool CByteCode::Store(float x)
{
	if (!CheckSpace(4)) return false;

	int y;
	memcpy(&y, &x, 4);
	// Do I really need to do this?
	m_CodeBuffer[m_Index++] = (U8)((y & 0xFF000000) >> 24);
	m_CodeBuffer[m_Index++] = (U8)((y & 0x00FF0000) >> 16);
	m_CodeBuffer[m_Index++] = (U8)((y & 0x0000FF00) >>  8);
	m_CodeBuffer[m_Index++] = (U8) (y & 0x000000FF);

	return true;
}

void CByteCode::Read(float& x)
{
	int y = (((U8)(m_CodeBuffer[m_Index])     << 24) |
		     ((U8)(m_CodeBuffer[m_Index + 1]) << 16) |
		     ((U8)(m_CodeBuffer[m_Index + 2]) <<  8) |
		      (U8) m_CodeBuffer[m_Index + 3]);

	memcpy(&x, &y, 4);
	m_Index += 4;
}

bool CByteCode::Store(LPCSTR x, int length)
{
	// Don't store NULL terminator
	if (!CheckSpace(length)) return false;
	memcpy(m_CodeBuffer + m_Index, x, length);
	m_Index += length;
	return true;
}

bool CByteCode::Store(LPCWSTR x, int length)
{
	// Don't store NULL terminator
	if (!CheckSpace(length)) return false;
	// Darn Unicode!  We store only 8-bit ASCII chars
	for (int idx = 0; idx < length; ++idx) {
		m_CodeBuffer[m_Index++] = (U8)(x[idx]);
	}
	return true;
}

char CByteCode::NextChar(bool bConvertCRLF)
{
	m_ch0 = m_ch1;

	if (m_ch1 != EOF) {
		while ((m_ch1 = *m_Text++) == 24);	// Strip out control-X from text
		if (m_ch1 == 0) {
			m_ch1 = EOF;
		}
	}

	// Convert all forms of EOL (ie \r\n, \r, and \n) to just \n
	if (bConvertCRLF && (m_ch1 == '\r')) {
		m_ch1 = '\n';
		if (*m_Text == '\n') {
			++m_Text;	// Skip it
		}
	}

	return m_ch0;
}

bool CByteCode::_StartNow(const char *text, int textLength)
{
	// Free any existing code buffer...
	if (m_CodeBuffer != NULL) free(m_CodeBuffer);

	// Guess initial size to use for m_CodeBuffer and create it
	m_CodeBufferLength = textLength * 2;
	m_CodeBuffer = (U8*)malloc(m_CodeBufferLength);
	if (m_CodeBuffer == NULL) {
		m_LastError = _T("Unable to allocate space for code buffer");
		return false;
	}

	m_Text = text;	// Pointer to 8-bit ASCII code to compile
	m_ch0 = ' ';
	m_ch1 = ' ';
	m_bInQuotedString = false;
	m_StringResult.Empty();
	m_IntegerResult = 0;
	m_FloatResult = 0.0f;
	m_Index = 0;
	m_TokenType = tokUndefined;
	m_LineNumber = 1;

	bool bDone = false;
	while (!bDone) {
		if (m_TokenType == tokEOL) {
			++m_LineNumber;	// Do this before the rest of NextToken() !!!
		}

		// Remove white spaces and look for EOL marker
		while ((m_ch0 <= ' ') && (m_ch0 != '\n') && (m_ch0 != EOF)) {
			NextChar();
		}

		if (m_ch0 == '\n') {
			NextChar();
			if (!Store(tokEOL)) return false;
			++m_LineNumber;	// Used in a few compile time error reports
		}
		else if ((m_ch0 == '_') || isalpha(m_ch0)) {
			if (!ParseIdent()) return false;
		}
		else if ((m_ch0 == '0') && ((m_ch1 == 'x') || (m_ch1 == 'X'))) {
			if (!ParseHexNumber()) return false;
		}
		else if (isdigit(m_ch0)) {
			if (!ParseNumber()) return false;
		}
		else if (m_ch0 == '"') {
			if (!ParseQuotedString()) return false;
		}
		else if ((m_ch0 == '/') && (m_ch1 == '*')) {
			if (!ParseBlockComment()) return false;
		}
		else if ((m_ch0 == '#') || ((m_ch0 == '/') && (m_ch1 == '/'))) {
			if (!ParseLineComment()) return false;
		}
		else if (m_ch0 == EOF) {
			// We are done compiling!
			// Re-adjust the size of the code buffer (it's probably to big)
			if (!Store(tokEOF)) return false;
			m_CodeBufferLength = m_Index;
			m_CodeBuffer = (U8*)realloc(m_CodeBuffer, m_CodeBufferLength);
			Rewind();
			bDone = true;
		}
		else {
			if (!ParseSpecial()) return false;
		}
	}
	return true;
}

bool CByteCode::ParseIdent()
{
	char buf[128];	// Only store first 127 chars of ident name
	int idx = 0;

	while (isalnum(m_ch0) || (m_ch0 == '_')) {
		if (idx < 127) buf[idx++] = m_ch0;
		NextChar();
	}

	if (m_ch0 == '$') {
		// These $ is allowed as the final char of an ident
		if (idx < 127) buf[idx++] = m_ch0;
		NextChar();
	}

	buf[idx] = 0;	// Don't forget to NULL terminate!

	m_TokenType = StringToTokenType(buf);	// Check to see if we have a keyword, etc.
	if (!Store(m_TokenType)) return false;	// Store the byte code representing the token type

	if (m_TokenType == tokIdent) {
		// It is a variable name...
		// First store the length of the string, and then the string without NULL term
		if (!Store(idx) || !Store(buf, idx)) return false;
	}
	return true;
}

#define VALID_HEX_DIGIT(ch) \
	(isdigit(ch) || (((ch) >= 'A') && ((ch) <= 'F')) || (((ch) >= 'a') && ((ch) <= 'f')))


bool CByteCode::ParseHexNumber()
{
	// Hex numbers start with 0x (eg. 0x4FA1)
	NextChar();	// Eat the zero
	NextChar();	// Eat the 'x'
	if (!VALID_HEX_DIGIT(m_ch0)) {
		m_LastError.Format(_T("Invalid hexadecimal number around line %d"), m_LineNumber);
		return false;
	}

	int value = 0;

	while (isalnum(m_ch0)) {
		if (isdigit(m_ch0)) {
			value = (value << 4) | (m_ch0 - '0');
		}
		else if ((m_ch0 >= 'A') && (m_ch0 <= 'F')) {
			value = (value << 4) | (m_ch0 - 'A' + 10);
		}
		else if ((m_ch0 >= 'a') && (m_ch0 <= 'f')) {
			value = (value << 4) | (m_ch0 - 'a' + 10);
		}
		else {
			m_LastError.Format(_T("Invalid hexadecimal number around line %d"), m_LineNumber);
			return false;
		}
		NextChar();
	}

	if (!Store(tokInteger)) return false;
	if (!Store(value)) return false;

	return true;
}

bool CByteCode::ParseNumber()
{
	bool bSeenDot = false;

	char buf[64];
	int idx = 0;

	m_IntegerResult = 0;
	m_FloatResult = 0.0f;
	m_TokenType = tokInteger;	// Assume integer until see decimal

	while (isdigit(m_ch0) || (m_ch0 == '.') || (m_ch0 == 'e') || (m_ch0 == 'E')) {
		if (idx >= 63) {
			m_LastError.Format(_T("Too many digits in literal number on line %d"), m_LineNumber);
			return false;
		}
		else if (m_ch0 == '.') {
			if (bSeenDot) {
				m_LastError.Format(_T("Not a valid number (multiple decimal points found)")
					_T(" on line %d"), m_LineNumber);
				return false;
			}
			bSeenDot = true;
			m_TokenType = tokFloat;
		}
		else if ((m_ch0 == 'e') || (m_ch0 == 'E')) {
			m_TokenType = tokFloat;
			buf[idx++] = m_ch0;
			NextChar();
			if ((m_ch0 == '+') || (m_ch0 == '-')) {
				buf[idx++] = m_ch0;
				NextChar();
			}
			if (!isdigit(m_ch0)) {
				m_LastError.Format(_T("Not a valid number (require digit or +/- after 'E')")
					_T(" on line %d"), m_LineNumber);
				return false;
			}
			while (isdigit(m_ch0)) {
				buf[idx++] = m_ch0;
				NextChar();
			}
			break;	// Nothing else is valid after this, so let's just break now
		}
		buf[idx++] = m_ch0;
		NextChar();
	}
	buf[idx] = 0;

	if (!Store(m_TokenType)) return false;

	if (m_TokenType == tokInteger) {
		if (!Store(atoi(buf))) return false;
	}
	else if (m_TokenType == tokFloat) {
		#ifdef __WINCE__
			TCHAR* ptr;
			TString temp = buf;
			if (!Store((float)_tcstod(temp, &ptr))) return false;
		#else
			if (!Store((float)atof(buf))) return false;
		#endif
	}
	return true;
}

bool CByteCode::ParseQuotedString()
{
	int startLine = m_LineNumber;

	char* buf = (char*)malloc(512);
	int idx = 0, bufSize = 512;

	NextChar();	// Remove quote char
	while (m_ch0 != '"') {
		// Make sure we aren't about to overrun the buffer
		if (idx >= bufSize - 1) {
#pragma message("********** Possible bug when quoted string exceeds original buffer size ********")
			bufSize += 512;
			buf = (char*)realloc(buf, bufSize);
			if (buf == NULL) {
				m_LastError = _T("remalloc failed while parsing quoted string");
				return false;
			}
		}

		// Convert accent marks to quotation marks
		buf[idx++] = m_ch0 == '`' ? '"' : m_ch0;
		NextChar(false);
		// We are allowing a quoted string to span multiple lines
		// since the ascii data files might contain notes with carriage returns
		if (m_ch0 == EOF) {
			buf[100] = '\0';
			m_LastError.Format(
				_T("Reached EOF before closing quote for string starting at line %d\n\n%s"),
				startLine, buf);
			free(buf);
			return false;
		}
	}
	NextChar();		// Remove quote char
	buf[idx] = 0;	// Terminate

	if (!Store(tokQuotedString) || !Store(idx) || !Store(buf, idx)) {
		free(buf);
		return false;
	}
	free(buf);
	return true;
}

bool CByteCode::ParseBlockComment()
{
	while (!((m_ch0 == '*') && (m_ch1 == '/')) && (m_ch1 != EOF)) {
		if (m_ch0 == '\n') {
			if (!Store(tokEOL)) return false;	// Keep line numbers in sync
		}
		NextChar();
	}
	NextChar();		// Eat the *
	NextChar();		// Eat the /
	NextToken();
	return true;
}

bool CByteCode::ParseLineComment()
{
	while (true) {
		switch (m_ch0) {
			case '\n':
				if (!Store(tokEOL)) return false;
				NextChar();
				return true;
			case EOF:
				return true;	// Pass EOF up the chain...
			default:
				NextChar();
		}
	}
	return false;	// Never reached...
}

bool CByteCode::ParseSpecial()
{
	char buf[256];	// Max size of persistent comment text
	int idx = 0;

	switch (m_ch0) {
		case '(':	m_TokenType = tokOpenParen; break;
		case ')':	m_TokenType = tokCloseParen; break;
		case '[':	if (m_ch1 == ':') {
						m_TokenType = tokStartChunk;
						NextChar();
					}
					else m_TokenType = tokOpenBracket;
					break;
		case ']':	m_TokenType = tokCloseBracket; break;
		case '{':	m_TokenType = tokOpenBrace; break;
		case '}':	m_TokenType = tokCloseBrace; break;
		case ':':	if (m_ch1 == ']') {
						m_TokenType = tokEndChunk;
						NextChar();
					}
					else m_TokenType = tokColon;
					break;
		case ';':	m_TokenType = tokSemicolon;	break;
		case ',':	m_TokenType = tokComma; break;
		case '.':	m_TokenType = tokDot; break;
		case '#':	m_TokenType = tokPound; break;
		case '\'':	m_TokenType = tokTick; break;
		case '^':	m_TokenType = tokPower; break;
		case '"':	m_TokenType = tokDoubleQuote; break;
		case '?':	m_TokenType = tokQuestion; break;
		case '~':	m_TokenType = tokTilde; break;
		case '%':	m_TokenType = tokPercent; break;
		case '!':	if (m_ch1 == '=') {
						m_TokenType = tokNotEquals;
						NextChar();
					}
					else m_TokenType = tokBitNot;
					break;
		case '-':	if (m_ch1 == '=') {
						m_TokenType = tokMinusEqual;
						NextChar();
					}
					else if (m_ch1 == '>') {
						m_TokenType = tokRange;
						NextChar();
					}
					else m_TokenType = tokMinus;
					break;
		case '+':	if (m_ch1 == '=') {
						m_TokenType = tokPlusEqual;
						NextChar();
					}
					else m_TokenType = tokPlus;
					break;
		case '*':	if (m_ch1 == '=') {
						m_TokenType = tokTimesEqual;
						NextChar();
					}
					else m_TokenType = tokTimes;
					break;
		case '/':	if (m_ch1 == '=') {
						m_TokenType = tokDivideEqual;
						NextChar();
					}
					else m_TokenType = tokDivide;
					break;
		case '\\':	if (m_ch1 == '\\') {
						m_TokenType = tokPersistComment;
						NextChar();	// Move second backslash to m_ch0
						while ((m_ch1 != '\n') && (m_ch1 != EOF)) {
							if (idx < 255) buf[idx++] = NextChar();
						}
						buf[idx] = 0;
					}
					else m_TokenType = tokBackslash;
					break;
		case '=':	if (m_ch1 == '=') {
						m_TokenType = tokBooleanEquals;
						NextChar();
					}
					else m_TokenType = tokAssignment;
					break;
		case '>':	if (m_ch1 == '=') {
						m_TokenType = tokGreaterEqual;
						NextChar();
					}
					else if (m_ch1 == '>') {
						m_TokenType = tokLeftShift;
						NextChar();
					}
					else if (m_ch1 == '<') {
						m_TokenType = tokNotEqualsCase;
						NextChar();
					}
					else m_TokenType = tokGreater;
					break;
		case '<':	if (m_ch1 == '=') {
						m_TokenType = tokLessEqual;
						NextChar();
					}
					else if (m_ch1 == '>') {
						m_TokenType = tokNotEquals;
						NextChar();
					}
					else if (m_ch1 == '<') {
						m_TokenType = tokRightShift;
						NextChar();
					}
					else m_TokenType = tokLess;
					break;
		default:
			m_LastError.Format(_T("Unknown special character ('%c') found while compiling text near line %d"),
				m_ch0, m_LineNumber);
			return false;
	}
	NextChar();

	// Store it in the code buffer...
	if (!Store(m_TokenType)) return false;
	if (m_TokenType == tokPersistComment) {
		if (!Store(idx) || !Store(buf, idx)) return false;
	}
	return true;
}

///////////////////////////////////////////////////////////
// Serialization of the byte code stream...
///////////////////////////////////////////////////////////

U8* CByteCode::SerializeIn(U8* inBuffer, CByteCode** result)
{
	ASSERT(*result == NULL);
	CByteCode* code = new CByteCode();

	inBuffer = SerializeInTString(inBuffer, code->m_ParentModule);
	inBuffer = SerializeInInt(inBuffer, (int&)code->m_CodeBufferLength);
	code->m_CodeBuffer = (U8*)malloc(code->m_CodeBufferLength);
	memcpy(code->m_CodeBuffer, inBuffer, code->m_CodeBufferLength);
	inBuffer += code->m_CodeBufferLength;

	*result = code;
	return inBuffer;
}

void CByteCode::SerializeOut(CFile* outFile)
{
	SerializeOutTString(outFile, m_ParentModule);
	SerializeOutInt(outFile, m_CodeBufferLength);
	outFile->Write(m_CodeBuffer, m_CodeBufferLength);
}

///////////////////////////////////////////////////////////
// Construction and destruction...
///////////////////////////////////////////////////////////

CByteCode::CByteCode()
:m_CodeBufferLength(0), m_CodeBuffer(NULL), m_Index(-1)
{
	// Assumptions made in compiled code
	ASSERT(sizeof(int) == 4);
	ASSERT(sizeof(float) == 4);

	m_LineNumber = 1;
	m_LineOffset = 0;
	m_TokenType = tokUndefined;
	m_StringResult.Empty();
	m_IntegerResult = 0;
	m_FloatResult = 0.0f;
	m_bInQuotedString = false;
}

CByteCode::~CByteCode()
{
	free(m_CodeBuffer);
}

CByteCode::CByteCode(const CByteCode& copy)
{
	if (&copy != this) {
		m_Path = copy.m_Path;
		m_ParentModule = copy.m_ParentModule;
		m_CodeBufferLength = copy.m_CodeBufferLength;
		m_CodeBuffer = (U8*)malloc(m_CodeBufferLength);
		memcpy(m_CodeBuffer, copy.m_CodeBuffer, m_CodeBufferLength);
		Rewind();
	}
}

CByteCode::CByteCode(CByteCode& in, TokenTypes headTok, TokenTypes tailTok)
{
	// This constructor is generally called when we want to extract
	// the code for a Dialect declared function into a sub-stream which
	// can be stored and executed at anytime and anywhere
	// Hence, headTok is usually tokFunc and tailTok is usually tokEndFunc
	// IMPORTANT: This function is used during execution of code ("in")
	// and therefore when this function leaves, "in" must be set correctly
	// to the token tailTok.  Possibly make this in non-constructor form?

	int iTemp;
	int depth = 1;	// It is assumed that headTok was already eaten

	m_Path = in.m_Path;
	m_ParentModule = in.m_ParentModule;	// Module is the same as in's..
	m_LineOffset = in.LineNumber() - 1;

	/*
	 *	The compiled buffer size is always less (or equal to) the
	 *	size of the code buffer in the copied CByteCode "in"
	 */
	m_Index = 0;
	m_CodeBufferLength = in.m_CodeBufferLength;
	m_CodeBuffer = (U8*)malloc(m_CodeBufferLength);

	while (true) {
		// Check for nesting...
		if (in.m_TokenType == headTok) ++depth;
		else if (in.m_TokenType == tailTok) --depth;

		if (depth == 0) break;	// We are done...

		// First store the token
		Store(in.m_TokenType);

		// Then store any extra info needed
		switch (in.m_TokenType) {
			case tokIdent:
			case tokQuotedString:
			case tokPersistComment:
				// We have an int for length and then chars...
				iTemp = in.m_StringResult.GetLength();
				Store(iTemp);
				Store(in.m_StringResult, iTemp);
				break;
			case tokInteger:
				Store(in.m_IntegerResult);
				break;
			case tokFloat:
				Store(in.m_FloatResult);
				break;
			case tokEOF:
				// This is actually an error condition!
				goto endCopy;
		}
		++in;	// Get next token
	}

endCopy:
	m_CodeBufferLength = m_Index;
	m_CodeBuffer = (U8*)realloc(m_CodeBuffer, m_Index);
	Rewind();
}

///////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////

void CByteCode::NextToken()
{
	TCHAR* buf;
	int iTemp, idx;
	m_StringResult.Empty();
	m_IntegerResult = 0;
	m_FloatResult = 0.0f;

	if (m_Index >= m_CodeBufferLength) {
		m_TokenType = tokEOF;
		return;
	}

	if (m_TokenType == tokEOL) {
		// Make sure we don't increment line number when we get the tokEOL
		// We must do it after we get the byte code after the tokEOL
		++m_LineNumber;
	}

	// Type casting to the enum TokenTypes is an undefined proccess
	// It might not work correctly, hence a lookup table...
	m_TokenType = ByteCodeToType[m_CodeBuffer[m_Index++]];

	switch (m_TokenType) {
		case tokQuotedString:
		case tokIdent:
		case tokPersistComment:
			Read(iTemp);	// Read length of string to follow

			buf = m_StringResult.GetBuffer(iTemp);
			for (idx = 0; idx < iTemp; ++idx) {
				buf[idx] = (TCHAR)(m_CodeBuffer[m_Index++]);
			}
			m_StringResult.ReleaseBuffer(iTemp);
			break;
		case tokInteger:
			Read(m_IntegerResult);
			break;
		case tokFloat:
			Read(m_FloatResult);
			break;
		case tokEOL:
			// Don't incremement m_LineNumber here!
			break;
	}
}

///////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////
///////////////////////////////////////////////////////////

TString CByteCode::ModuleName()
{
	return m_ParentModule;
}

TString CByteCode::ModulePath()
{
	return m_Path;
}

int CByteCode::LineNumber() const
{
	return m_LineNumber + m_LineOffset;
}

int CByteCode::LineNumberNoOffset() const
{
	return m_LineNumber;
}

TString CByteCode::GetLastError() const
{
	return m_LastError;
}

int CByteCode::Length() const
{
	return m_CodeBufferLength;
}

const U8* CByteCode::Data() const
{
	return m_CodeBuffer;
}

void CByteCode::Seek(long pos, int lineNum)
{
	m_Index = pos;
	m_LineNumber = lineNum;

	// ALWAYS set m_TokenType to tokUndefined before calling NextToken()
	// if we are moving the position of the stream after changing m_Index.
	// Otherwise, if m_TokenType is tokEOL, then line numbers will get
	// screwed up.
	m_TokenType = tokUndefined;
	NextToken();
}

void CByteCode::Rewind()
{
	m_Index = 0;
	m_LineNumber = 1;
	m_StringResult.Empty();
	m_IntegerResult = 0;
	m_FloatResult = 0.0f;
	m_TokenType = tokUndefined;	// Do this before NextToken()
	NextToken();				// or else linenumbers could get messed up!!!
}

long CByteCode::Position() const
{
	return m_Index;
}

///////////////////////////////////////////////////////////
//  Attempt to turn byte codes back into readable text
///////////////////////////////////////////////////////////

TString CByteCode::__unparse__()
{
	// Save current info
	int oldIndex = m_Index;
	int oldLineNumber = m_LineNumber;
	TokenTypes oldTokenType = m_TokenType;
	TString oldStringResult = m_StringResult;
	int oldIntegerResult = m_IntegerResult;
	float oldFloatResult = m_FloatResult;

	TString result, tempStr;
	Rewind();

	int tab = 1, tabIdx;
	bool bDoTab = true;
#define DO_TAB	{if (bDoTab) for (tabIdx = 0; tabIdx < tab; ++tabIdx) result += '\t'; bDoTab = false;}
#define INC_TAB	++tab;
#define DEC_TAB	--tab;

	while (m_TokenType == tokEOL) {
		NextToken();
	}

	while (m_TokenType != tokEOF) {
		switch (m_TokenType) {
			case tokEOL:
				result += "\r\n";
				bDoTab = true;
				break;
			case tokIf:
				DO_TAB;
				result += _T("if ");
				INC_TAB;
				break;
			case tokEndIf:
				DEC_TAB;
				DO_TAB;
				result += _T("endif ");
				break;
			case tokElse:
				DEC_TAB;
				DO_TAB;
				result += _T("else ");
				INC_TAB;
				break;
			case tokElseIf:
				DEC_TAB;
				DO_TAB;
				result += _T("elif ");
				INC_TAB;
				break;
			case tokWhile:
				DO_TAB;
				result += _T("while ");
				INC_TAB;
				break;
			case tokEndWhile:
				DEC_TAB;
				DO_TAB;
				result += _T("endwhile ");
				break;
			case tokFor:
				DO_TAB;
				result += _T("for ");
				INC_TAB;
				break;
			case tokEndFor:
				DEC_TAB;
				DO_TAB;
				result += _T("endfor ");
				break;
			case tokForEach:
				DO_TAB;
				result += _T("foreach ");
				INC_TAB;
				break;
			case tokEndForEach:
				DEC_TAB;
				DO_TAB;
				result += _T("endforeach ");
				break;
			case tokWith:
				DO_TAB;
				result += _T("with ");
				INC_TAB;
				break;
			case tokEndWith:
				DEC_TAB;
				DO_TAB;
				result += _T("endwith ");
				break;
			case tokTry:
				DO_TAB;
				result += _T("try ");
				INC_TAB;
				break;
			case tokEndTry:
				DEC_TAB;
				DO_TAB;
				result += _T("endtry ");
				break;
			case tokCatch:
				DEC_TAB;
				DO_TAB;
				result += _T("catch ");
				INC_TAB;
				break;
			case tokFinally:
				DEC_TAB;
				DO_TAB;
				result += _T("finally ");
				INC_TAB;
				break;
			case tokIdent:
				DO_TAB;
				result += m_StringResult + _T(" ");
				break;
			case tokInteger:
				DO_TAB;
				tempStr.Format(_T("%d "), m_IntegerResult);
				result += tempStr;
				break;
			case tokFloat:
				DO_TAB;
				/*#ifdef __WINCE__
					{
						char buf[64];
						_gcvt(m_FloatResult, 8, buf);
						int idx = 0;
						while (buf[idx] != 0) {
							result += (TCHAR)buf[idx++];
						}
					}
				#else*/
					tempStr.Format(_T("%g"), m_FloatResult);
					result += tempStr;
				//#endif
				break;
			case tokQuotedString:
				DO_TAB;
				{
					TString unparsed = m_StringResult;
					for (int idx = 0; idx < unparsed.GetLength(); ++idx) {
						if (unparsed[idx] == '"') {
							unparsed.SetAt(idx, '`');
						}
					}
					tempStr.Format(_T("\"%s\" "), (LPCTSTR)unparsed);
				}
				result += tempStr;
				break;
			case tokPersistComment:
				DO_TAB;
				result += _T("\\\\");
				result += m_StringResult;
				break;
			default:
				DO_TAB;
				result += TokenTypeToString(m_TokenType) + _T(" ");
		}
		NextToken();
	}

	// Restore original info
	m_Index = oldIndex;
	m_LineNumber = oldLineNumber;
	m_TokenType = oldTokenType;
	m_StringResult = oldStringResult;
	m_IntegerResult = oldIntegerResult;
	m_FloatResult = oldFloatResult;

	return result;
}

///////////////////////////////////////////////////////////
// Turn text into byte code
///////////////////////////////////////////////////////////

bool CByteCode::CompileText(LPCTSTR text, LPCTSTR moduleName)
{
	return _Compile(text, moduleName, _T(""));
}

#ifdef __WINCE__
#	pragma warning(disable:4509)	// SEH and C++ destructor warning
#endif
bool CByteCode::CompileFile(TString filename)
{
	TFile file;
	_TRY {
		file.Open(filename, TFile::read);
	}
	_CATCH_ALL {
		m_LastError.Format(_T("Failed to open file: %s"), (LPCTSTR)filename);
		return false;
	}

	int length = file.GetLength();
	char* text = new char[length + 2];		// 2 == in case of Unicode
	file.Read((void*)text, length);			//
	text[length] = 0; text[length + 1] = 0;	// NULL terminate

	/*
	 *	filename is something like "C:\\This is a test\\of something\\testcode.txt"
	 *	and we want to extract just "testcode".  For Unix compatibility we would also
	 *	need to check for '/'
	 */
	TString modName;
	modName = filename.Right(filename.GetLength() - filename.ReverseFind(DIR_SEPARATOR) - 1);
	modName = modName.Left(modName.ReverseFind('.'));

	bool bResult;
	if ((length > 1) && ((text[0] == 0) || (text[1] == 0))) {
		// Guessing it's an Unicode file...
		bResult = _Compile((LPCWSTR)text, modName, GetFullPath(filename));
	}
	else {
		// Guessing it's an ASCII file...
		bResult = _Compile((LPCSTR)text, modName, GetFullPath(filename));
	}
	delete[] text;

	return bResult;
}
#ifdef __WINCE__
#	pragma warning(default:4509)	// SEH and C++ destructor warning
#endif

///////////////////////////////////////////////////////////
// Read and write byte code to and from files, etc.
///////////////////////////////////////////////////////////

#ifdef __WINCE__
#	pragma warning(disable:4509)	// SEH and C++ destructor warning
#endif
bool CByteCode::SaveToFile(TString filename)
{
	U8 temp[3] = {'d' + 128, 'c' + 128, FILE_VERSION_BYTE};

	//TFile file;
	//_TRY {
	//	file.Open(filename, TFile::write | TFile::create);
	//}
	//_CATCH_ALL {
	//	m_LastError.Format(_T("Failed to create file: %s"), (LPCTSTR)filename);
	//	return false;
	//}
	CFile file;
	if (!file.Open(filename, CFile::modeCreate | CFile::modeWrite | CFile::shareDenyWrite | CFile::typeBinary)) {
		m_LastError.Format(_T("Failed to create file: %s"), (LPCTSTR)filename);
		return false;
	}

	// First write a magic header
	file.Write(temp, 3);
	SerializeOut(&file);
	file.Close();
	return true;
}
#ifdef __WINCE__
#	pragma warning(default:4509)	// SEH and C++ destructor warning
#endif

#ifdef __WINCE__
#	pragma warning(disable:4509)	// SEH and C++ destructor warning
#endif
bool CByteCode::LoadFromFile(TString filename)
{
	TFile file;
	_TRY {
		file.Open(filename, TFile::read);
	}
	_CATCH_ALL {
		m_LastError.Format(_T("Failed to open file: %s"), (LPCTSTR)filename);
		return false;
	}

	int filesize = file.GetLength();
	U8* buffer = new U8[filesize];
	file.Read(buffer, filesize);
	file.Close();

	bool bResult = LoadFromMemory(buffer);
	delete[] buffer;

	if (!bResult) {
		m_LastError.Format(
			_T("Attempt to load precompiled file '%s' failed due to incorrect header. ")
			_T("Possible file format version change.  If you have the text version of the ")
			_T("file in the script path, you could delete the precompiled file which ")
			_T("should cause it to be recreated using the current file format."),
			(LPCTSTR)filename);
	}
	else {
		m_Path = GetFullPath(filename);
	}

	return bResult;
}
#ifdef __WINCE__
#	pragma warning(default:4509)	// SEH and C++ destructor warning
#endif

bool CByteCode::LoadFromMemory(U8* buffer)
{
	if ((buffer[0] == 'd' + 128) && (buffer[1] == 'c' + 128) && (buffer[2] == FILE_VERSION_BYTE)) {
		// This is code from SerializeIn, except we don't want to
		// create a new CByteCode as SerializeIn does, use this one
		if (m_CodeBuffer != NULL) {
			free(m_CodeBuffer);
			m_CodeBuffer = NULL;
		}
		buffer = SerializeInTString(buffer + 3, m_ParentModule);
		buffer = SerializeInInt(buffer, (int&)m_CodeBufferLength);
		m_CodeBuffer = (U8*)malloc(m_CodeBufferLength);
		memcpy(m_CodeBuffer, buffer, m_CodeBufferLength);
		Rewind();
		return true;
	}
	else {
		m_LastError = _T("Attempt to load precompiled memory file failed: header incorrect");
		return false;
	}
}

///////////////////////////////////////////////////////////
// Stuff that is typically inlined in header...
///////////////////////////////////////////////////////////

/*CByteCode::operator TokenTypes() const
{
	return m_TokenType;
}

CByteCode::operator LPCTSTR() const
{
	return (LPCTSTR)m_StringResult;
}

CByteCode::operator int()
{
	return m_IntegerResult;
}

CByteCode::operator float()
{
	return m_FloatResult;
}

CByteCode& CCodeStream::operator++()
{
	// Prefix increment   eg. ++token
	NextToken();
	return *this;
}*/

CByteCode CByteCode::operator++(int)
{
	// Postfix increment  eg. token++
	// THIS DOESN"T WORK CORRECTLY!!!  WE JUST DO PREFIX INCREMENTING!!!!
	ASSERT(false);
	NextToken();
	return *this;
}
